from django.shortcuts import render, get_object_or_404, redirect
from django.http import JsonResponse, HttpResponseBadRequest
from django.urls import reverse
from django.core.paginator import Paginator
from django.db.models import Q
from django.utils import timezone
from django import forms
from django.db.models.deletion import ProtectedError
from django.conf import settings
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from django.views.decorators.cache import never_cache
from django.contrib.auth.decorators import login_required, permission_required
from django.db import transaction
from .models import AIProvider, Scene, Mailbox, EmailMessage, process_extraction_and_merge, ExtractionRun, ExtractionResult, Shipment

import re
import os
import email
import traceback
import imaplib
import socket
import urllib.parse
from email.header import decode_header
from email.utils import parsedate_to_datetime
import uuid
import json
import ast
from typing import Optional, List
import logging
import requests

logger = logging.getLogger(__name__)
# 设置日志级别
logger.setLevel(logging.DEBUG)

# 创建控制台处理器并设置级别
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.DEBUG)

# 设置日志格式
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
console_handler.setFormatter(formatter)

# 给记录器添加处理器
logger.addHandler(console_handler)

_FORWARD_PATTERMS = [
            r'-----Original Message-----', r'On\s.+?\s+wrote:\s*$',
            r'^From:\s+.*$', r'^To:\s+.*$', r'^Sent:\s+.*$', r'^Subject:\s+.*$',
            r'-{5,}', r'_{5,}', r'\*{5,}', r'^>\s+.*', r'^转发：\s*.*',
            r'^发件人\s*.*', r'^日期：\s*.*', r'^-----邮件原件-----',
            r'^-----原始邮件-----', r'^INTERNAL', r'发件人:', r'blockquote',
            r'Original Message', r'Best Regards', r'best regards',
            r'Tks & B.Rgds', r'Thanks and Best Regards', r'吴洪兵',
            r'From:', r'中国外运华东有限公司储运分公司', r'Your Sincerely',
            r'敬祝', r'此致敬意', r'田珩玮', r'从现在开始所有欧盟 _国家ETD-3天截提单'
]

class SceneForm(forms.ModelForm):
    class Meta:
        model = Scene
        fields = [
            'name', 'description', 'email_type', 'fields_to_extract', 'prompt_template',
            'ai_provider', 'enabled', 'priority', 'update_policy', 'alt_match_priority',
            'required_fields_for_update', 'allow_create_without_po', 'update_fields_whitelist'
        ]
        widgets = {
            'description': forms.Textarea(attrs={'rows': 2}),
            'fields_to_extract': forms.Textarea(attrs={'rows': 2, 'placeholder': '以逗号分隔，如: booking_no,bl_no,eta'}),
            'prompt_template': forms.Textarea(attrs={'rows': 6, 'placeholder': '提示词模板，支持变量'}),
            'alt_match_priority': forms.TextInput(attrs={'placeholder': '如: bl_no,container_no,booking_no'}),
            'required_fields_for_update': forms.TextInput(attrs={'placeholder': '更新前必须具备的字段，如: po_no,eta'}),
            'update_fields_whitelist': forms.TextInput(attrs={'placeholder': '限制可更新的字段白名单，如: eta,ata,container_no'}),
        }


class MailboxForm(forms.ModelForm):
    # 以多行文本方式编辑，每行一个正则；也兼容 JSON 数组
    forward_patterns = forms.CharField(
        required=False,
        widget=forms.Textarea(attrs={'rows': 3, 'placeholder': '每行一条正则，例如：^From:.*$\n^-{5,}$'}),
        label='清洗分隔规则（正则）'
    )

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        # 将实例中的 list 转为换行文本展示
        if self.instance and isinstance(getattr(self.instance, 'forward_patterns', None), list):
            self.initial.setdefault('forward_patterns', '\n'.join(self.instance.forward_patterns))

    def clean_forward_patterns(self):
        val = self.cleaned_data.get('forward_patterns')
        # 空值直接返回空列表
        if val in (None, ''):
            return []
        # 若已是列表，规范化返回
        if isinstance(val, list):
            return [str(x).strip() for x in val if str(x).strip()]
        s = str(val).strip()
        # 优先：JSON 数组
        try:
            parsed = json.loads(s)
            if isinstance(parsed, list):
                return [str(x).strip() for x in parsed if str(x).strip()]
        except Exception:
            pass
        # 其次：Python 字面量（支持 r'' 原始字符串）
        try:
            lit = ast.literal_eval(s)
            if isinstance(lit, list):
                return [str(x).strip() for x in lit if str(x).strip()]
        except Exception:
            pass
        # 回退：去除包裹的 [] 后按换行/逗号/分号拆分
        if s.startswith('[') and s.endswith(']'):
            s = s[1:-1]
        raw_parts = re.split(r'[\r\n;,]+', s)
        parts: List[str] = []
        for p in raw_parts:
            p = p.strip()
            if not p:
                continue
            # 去除可能的 r''/"" 或普通引号包裹
            if (len(p) >= 3 and p[0] in ('r', 'R') and p[1] in ("'", '"') and p[-1] == p[1]):
                p = p[2:-1].strip()
            elif (len(p) >= 2 and p[0] in ("'", '"') and p[-1] == p[0]):
                p = p[1:-1].strip()
            if p:
                parts.append(p)
        return parts

    class Meta:
        model = Mailbox
        fields = [
            'name', 'protocol', 'host', 'port', 'username', 'password_alias',
            'folder', 'use_ssl', 'enabled', 'forward_patterns'
        ]
        widgets = {
            'protocol': forms.TextInput(attrs={'placeholder': 'IMAP'}),
            'host': forms.TextInput(attrs={'placeholder': 'imap.example.com'}),
            'port': forms.NumberInput(attrs={'min': 1}),
            'username': forms.TextInput(attrs={'placeholder': 'user@example.com'}),
            'password_alias': forms.TextInput(attrs={'placeholder': '密钥别名，对应环境变量 MAILBOX_SECRET_<别名>'}),
            'folder': forms.TextInput(attrs={'placeholder': 'INBOX'}),
        }


def dashboard(request):
    return render(request, 'dashboard.html')


# ------------------ 场景 ------------------

def scenes(request):
    # 选择要编辑的场景（可为空表示新建）
    scene_id = request.GET.get('scene_id') or request.POST.get('scene_id')
    instance = None
    if scene_id:
        try:
            instance = Scene.objects.get(pk=scene_id)
        except Scene.DoesNotExist:
            instance = None

    saved = False
    error_msg = ''

    if request.method == 'POST':
        # 处理快捷操作：启用/禁用切换、删除
        op = request.POST.get('op')
        if op == 'toggle':
            if instance is not None:
                instance.enabled = not instance.enabled
                instance.save(update_fields=['enabled'])
                return redirect(f"/scenes?scene_id={instance.id}&saved=1")
            else:
                return redirect("/scenes?error=1")
        elif op == 'delete':
            if instance is not None:
                try:
                    instance.delete()
                    return redirect("/scenes?saved=1")
                except ProtectedError:
                    return redirect("/scenes?error=1")
            else:
                return redirect("/scenes?error=1")

        # 支持保存（创建/更新）
        form = SceneForm(request.POST, instance=instance)
        if form.is_valid():
            obj = form.save()
            saved = True
            # 重定向避免重复提交
            return redirect(f"/scenes?scene_id={obj.id}&saved=1")
        else:
            error_msg = '表单校验失败，请检查必填项与字段格式。'
    else:
        form = SceneForm(instance=instance)
        if request.GET.get('saved'):
            saved = True
        if request.GET.get('error'):
            error_msg = '操作失败（对象可能不存在或被引用，无法删除）。'

    # ---- 列表：搜索 + 筛选 + 分页 ----
    providers = AIProvider.objects.all().order_by('name')
    kw = (request.GET.get('kw') or '').strip()
    provider_id = (request.GET.get('provider') or '').strip()
    enabled_val = (request.GET.get('enabled') or '').strip()
    try:
        page = int(request.GET.get('page') or 1)
    except Exception:
        page = 1
    try:
        page_size = int(request.GET.get('page_size') or 10)
    except Exception:
        page_size = 10
    page_size = max(1, min(page_size, 100))

    scenes_qs = Scene.objects.select_related('ai_provider').all()
    if kw:
        scenes_qs = scenes_qs.filter(name__icontains=kw)
    if provider_id.isdigit():
        scenes_qs = scenes_qs.filter(ai_provider_id=int(provider_id))
    if enabled_val in ('1', '0'):
        scenes_qs = scenes_qs.filter(enabled=(enabled_val == '1'))

    scenes_qs = scenes_qs.order_by('priority', 'name')

    paginator = Paginator(scenes_qs, page_size)
    page_obj = paginator.get_page(page)

    ctx = {
        'form': form,
        'scenes': page_obj.object_list,
        'page_obj': page_obj,
        'providers': providers,
        'kw': kw,
        'provider_selected': provider_id,
        'enabled_selected': enabled_val,
        'page_size': page_size,
        'editing': instance is not None,
        'editing_scene': instance,
        'saved': saved,
        'error_msg': error_msg,
    }
    return render(request, 'scenes.html', ctx)


# ------------------ 邮箱 ------------------

def _decode_mime_header(value: str) -> str:
    if not value:
        return ''
    try:
        parts = decode_header(value)
        decoded = ''
        for frag, enc in parts:
            if isinstance(frag, bytes):
                try:
                    decoded += frag.decode(enc or 'utf-8', errors='ignore')
                except Exception:
                    decoded += frag.decode('utf-8', errors='ignore')
            else:
                decoded += frag
        return decoded
    except Exception:
        return value or ''


def _sanitize_filename(name: str) -> str:
    if not name:
        return uuid.uuid4().hex
    # 去掉尖括号和路径不安全字符
    name = name.strip().strip('<>')
    name = re.sub(r'[^A-Za-z0-9_.-]+', '_', name)
    return name or uuid.uuid4().hex


def _get_mailbox_password(mb: Mailbox) -> str:
    alias = (mb.password_alias or '').strip()
    if not alias:
        raise ValueError('未设置密码别名 password_alias')
    env_key = f"MAILBOX_SECRET_{alias}"
    password = os.environ.get(env_key)
    if not password:
        raise ValueError(f'未找到环境变量 {env_key}，请在 .env 或系统环境中配置邮箱密码/密钥')
    return password


def _imap_login(mb: Mailbox):
    password = _get_mailbox_password(mb)
    try:
        if mb.use_ssl:
            imap = imaplib.IMAP4_SSL(mb.host, mb.port)
        else:
            imap = imaplib.IMAP4(mb.host, mb.port)
        imap.login(mb.username, password)
        # 选择文件夹
        typ, _ = imap.select(mb.folder or 'INBOX', readonly=True)
        if typ != 'OK':
            try:
                imap.logout()
            except Exception:
                pass
            raise ValueError(f"无法选择文件夹: {mb.folder}")
        return imap
    except imaplib.IMAP4.error as e:
        raise ValueError(f"IMAP 登录失败: {str(e)}")
    except socket.gaierror:
        raise ValueError("无法解析邮箱服务器主机名（DNS错误）")
    except Exception as e:
        raise ValueError(f"连接邮箱失败: {str(e)}")


# ------------------ 场景 ------------------

def _test_mailbox_connection(mb: Mailbox):
    # 实际连接 IMAP 并尝试列出邮件以验证凭据与联通性
    imap = _imap_login(mb)
    try:
        # 简单做一次搜索验证有无权限
        typ, data = imap.search(None, 'ALL')
        if typ != 'OK':
            raise ValueError('IMAP 搜索失败')
    finally:
        try:
            imap.close()
        except Exception:
            pass
        try:
            imap.logout()
        except Exception:
            pass


def _get_charset(part):
    cs = part.get_content_charset() or part.get_charset()
    try:
        return str(cs) if cs else 'utf-8'
    except Exception:
        return 'utf-8'

try:
    from bs4 import BeautifulSoup  # type: ignore
except Exception:
    BeautifulSoup = None  # type: ignore
try:
    import html2text as _html2text  # type: ignore
except Exception:
    _html2text = None  # type: ignore

def _html_to_text_with_tables(html: str) -> str:
    if not html:
        return ''
    if BeautifulSoup is None:
        # 简单去标签降级
        return re.sub(r'<[^>]+>', '', html)
    soup = BeautifulSoup(html, 'html.parser')
    tables = soup.find_all('table')
    for table in tables:
        rows = table.find_all('tr')
        temp_table = []
        for row in rows:
            cells = row.find_all(['td', 'th'])
            temp_row = [cell.get_text(strip=True) for cell in cells]
            temp_row_str = ' | '.join(temp_row)
            temp_table.append(temp_row_str)
        temp_table_str = ' ; '.join(temp_table)
        temp_table_str = f"\n（以下数据是一个表格，';'是行的分隔符，'|'是列的分隔符,出现符号'###'说明此段表格数据结束）{temp_table_str}###\n"
        # 直接以纯文本替换表格节点
        table.replace_with(temp_table_str)
    if _html2text is not None:
        converter = _html2text.HTML2Text()
        converter.ignore_links = False
        text = converter.handle(str(soup))
    else:
        text = re.sub(r'<[^>]+>', '', str(soup))
    return text

def _clean_body_text(text: str, patterns: Optional[List[str]] = None) -> str:
    if not text:
        return ''
    # 先匹配转发/历史记录分隔标记，截断到第一处
    try:
        use_patterns = patterns if patterns else _FORWARD_PATTERMS
        pattern = re.compile('|'.join(use_patterns), flags=re.IGNORECASE | re.MULTILINE)
        m = pattern.search(text)
        if m:
            text = text[:m.start()].strip()
    except Exception:
        pass
    # 二次清理常见噪声
    for marker in ['发件人', 'From:', 'From：', '* * *']:
        idx = text.find(marker)
        if idx != -1:
            text = text[:idx].strip()
    # 去除零宽字符等
    return text.replace('\u200b', '').replace('\u0e07', '')


def _extract_bodies_and_attachments(msg, mailbox_id: int = None, save_selected: bool = False, forward_patterns: Optional[List[str]] = None):
    body_text = ''
    body_html = ''
    attachments = []

    def _decode_payload(part):
        payload = part.get_payload(decode=True)
        charset = _get_charset(part)
        try:
            return (payload or b'').decode(charset, errors='ignore')
        except Exception:
            return (payload or b'').decode('utf-8', errors='ignore')

    if msg.is_multipart():
        for part in msg.walk():
            if part.get_content_maintype() == 'multipart':
                continue
            ctype = part.get_content_type()
            disp = (part.get_content_disposition() or '').lower()
            filename = part.get_filename()
            payload_bytes = part.get_payload(decode=True)

            if filename:
                # 作为附件，但默认忽略图片类噪声；若命中关键词并允许保存，则仍保存并记录
                try:
                    name_decoded = _decode_mime_header(filename)
                except Exception:
                    name_decoded = filename
                is_image = ctype.startswith('image')
                size = len(payload_bytes) if isinstance(payload_bytes, (bytes, bytearray)) else 0

                saved_path = None
                if save_selected and mailbox_id and name_decoded and _should_save_attachment(name_decoded) and payload_bytes:
                    try:
                        saved_path = _save_attachment_file(mailbox_id, name_decoded, payload_bytes)
                    except Exception:
                        saved_path = None

                # 对于非图片附件，一律记录元数据；对于图片类，仅在命中保存策略时记录
                if not is_image:
                    item = {
                        'name': name_decoded,
                        'size': size,
                        'content_type': ctype,
                        'disposition': disp or 'attachment',
                    }
                    if saved_path:
                        item['saved_path'] = saved_path
                    attachments.append(item)
                else:
                    if saved_path:
                        attachments.append({
                            'name': name_decoded,
                            'size': size,
                            'content_type': ctype,
                            'disposition': disp or 'attachment',
                            'saved_path': saved_path,
                        })
            else:
                # 正文
                if ctype == 'text/plain':
                    if not body_text:
                        body_text = _decode_payload(part)
                elif ctype == 'text/html':
                    html = _decode_payload(part)
                    if not body_html:
                        body_html = html
                    # 将 HTML 转为更友好的纯文本（带表格展开）作为补充
                    text_from_html = _html_to_text_with_tables(html)
                    if not body_text:
                        body_text = text_from_html
                else:
                    # 其他类型，尝试当做文本兜底
                    text = _decode_payload(part)
                    if not body_text:
                        body_text = text
    else:
        ctype = msg.get_content_type()
        if ctype == 'text/plain':
            body_text = _decode_payload(msg)
        elif ctype == 'text/html':
            html = _decode_payload(msg)
            body_html = html
            body_text = _html_to_text_with_tables(html)
        else:
            body_text = _decode_payload(msg)

    # 正文清洗：去掉历史/转发/签名常见片段（支持自定义分隔规则）
    body_text = _clean_body_text(body_text, patterns=forward_patterns)

    return body_text, body_html, attachments


def _should_save_attachment(filename: str) -> bool:
    """根据 settings.ATTACHMENT_SAVE_KEYWORDS 判断是否需要保存附件到本地（大小写不敏感，子串匹配）。"""
    try:
        keywords = getattr(settings, 'ATTACHMENT_SAVE_KEYWORDS', []) or []
    except Exception:
        keywords = []
    name_lower = (filename or '').lower()
    for kw in keywords:
        if kw and kw.lower() in name_lower:
            return True
    return False


def _save_attachment_file(mailbox_id: int, filename: str, content: bytes) -> str:
    """将附件保存到本地，路径结构：EMAIL_STORAGE_DIR/attachments/<mailbox_id>/YYYY/MM/DD/<sanitized>_<uuid>.<ext>
    返回保存后的绝对路径。"""
    base_dir = getattr(settings, 'EMAIL_STORAGE_DIR', None)
    if not base_dir:
        # 兜底：与项目中 email_storage 一致
        base_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'email_storage')
    subroot = os.path.join(base_dir, 'attachments', str(mailbox_id))
    now = timezone.now()
    subdir = os.path.join(subroot, now.strftime('%Y'), now.strftime('%m'), now.strftime('%d'))
    os.makedirs(subdir, exist_ok=True)
    # 在原始文件名基础上加短随机后缀，尽量保留扩展名
    name = _sanitize_filename(filename or 'attachment')
    name_no_ext, ext = os.path.splitext(name)
    saved_name = f"{name_no_ext}_{uuid.uuid4().hex[:8]}{ext}"
    file_path = os.path.join(subdir, saved_name)
    with open(file_path, 'wb') as f:
        f.write(content or b'')
    return file_path

# 新增：保存原始邮件文件（.eml）
def _save_raw_email_file(mailbox_id: int, content: bytes) -> str:
    base_dir = getattr(settings, 'EMAIL_STORAGE_DIR', None)
    if not base_dir:
        base_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'email_storage')
    subroot = os.path.join(base_dir, 'raw', str(mailbox_id))
    now = timezone.now()
    subdir = os.path.join(subroot, now.strftime('%Y'), now.strftime('%m'), now.strftime('%d'))
    os.makedirs(subdir, exist_ok=True)
    saved_name = f"mail_{uuid.uuid4().hex[:16]}.eml"
    file_path = os.path.join(subdir, saved_name)
    with open(file_path, 'wb') as f:
        f.write(content or b'')
    return file_path

def _fetch_recent_emails(mb: Mailbox, limit: int = 20) -> dict:
    """拉取最近 limit 封邮件，保存到 EmailMessage，并解析正文与附件元数据（不落盘原文）。
    返回统计信息：{'fetched': x, 'skipped': y, 'created_ids': [...]}"""
    imap = _imap_login(mb)
    fetched = 0
    skipped = 0
    created_ids = []
    try:
        typ, data = imap.search(None, 'ALL')
        if typ != 'OK':
            raise ValueError('IMAP 搜索失败')
        ids = data[0].split() if data and data[0] else []
        logger.debug(f"Fetching {len(ids)} emails from {mb.name}...")
        if not ids:
            return {'fetched': 0, 'skipped': 0, 'created_ids': []}
        ids = ids[-limit:]
        for msg_id in ids:
            typ, msg_data = imap.fetch(msg_id, '(RFC822)')
            logger.debug(f"Fetching email {msg_id}...: {typ}")
            if typ != 'OK' or not msg_data:
                continue
            raw_bytes = msg_data[0][1]
            if not raw_bytes:
                continue
            try:
                msg = email.message_from_bytes(raw_bytes)
            except Exception:
                continue

            mid = (msg.get('Message-ID') or msg.get('Message-Id') or '').strip()
            if not mid:
                # 构造一个可重复但尽量稳定的替代ID（使用服务器UID + 随机）
                mid = f"uid-{msg_id.decode('utf-8', errors='ignore')}-{uuid.uuid4().hex[:8]}"

            if EmailMessage.objects.filter(message_id=mid).exists():
                skipped += 1
                continue

            subject = _decode_mime_header(msg.get('Subject'))
            from_addr = _decode_mime_header(msg.get('From'))
            to_addr = _decode_mime_header(msg.get('To'))
            try:
                dt = parsedate_to_datetime(msg.get('Date')) if msg.get('Date') else None
            except Exception:
                dt = None

            body_text, body_html, attach_list = _extract_bodies_and_attachments(msg, mb.id, False, forward_patterns=mb.forward_patterns)
            # 不保存原文到本地
            raw_path = ''
            obj = EmailMessage.objects.create(
                message_id=mid,
                subject=subject or '',
                from_addr=from_addr or '',
                to_addr=to_addr or '',
                date=dt,
                raw_path=raw_path,
                body_text=body_text or '',
                body_html=body_html or '',
                attachments=attach_list or [],
                mailbox=mb,
                status='pending'
            )
            fetched += 1
            created_ids.append(obj.id)
        return {'fetched': fetched, 'skipped': skipped, 'created_ids': created_ids}
    finally:
        try:
            imap.close()
        except Exception:
            pass
        try:
            imap.logout()
        except Exception:
            pass

# ------------------ 邮箱 ------------------

def mailboxes(request):
    mailbox_id = request.GET.get('mailbox_id') or request.POST.get('mailbox_id')
    instance = None
    if mailbox_id:
        try:
            instance = Mailbox.objects.get(pk=mailbox_id)
        except Mailbox.DoesNotExist:
            instance = None

    saved = False
    error_msg = ''
    test_msg = ''
    test_ok = None

    if request.method == 'POST':
        op = request.POST.get('op')
        # 行内操作：启用/禁用切换
        if op == 'toggle':
            if instance is not None:
                instance.enabled = not instance.enabled
                instance.save(update_fields=['enabled'])
                return redirect(f"/mailboxes?mailbox_id={instance.id}&saved=1")
            else:
                return redirect("/mailboxes?error=1")
        # 行内操作：删除
        if op == 'delete':
            if instance is not None:
                try:
                    instance.delete()
                    return redirect("/mailboxes?saved=1")
                except ProtectedError:
                    return redirect("/mailboxes?error=1")
            else:
                return redirect("/mailboxes?error=1")
        # 行内操作：测试连接
        if op == 'test':
            if instance is None:
                return redirect("/mailboxes?error=1")
            try:
                _test_mailbox_connection(instance)
                # 用查询串带回测试结果
                return redirect("/mailboxes?tested=1&ok=1&msg=" + urllib.parse.quote('连接成功'))
            except Exception as e:
                return redirect("/mailboxes?tested=1&ok=0&msg=" + urllib.parse.quote(str(e)))
        # 行内操作：拉取邮件
        if op == 'fetch':
            if instance is None:
                return redirect("/mailboxes?error=1")
            try:
                try:
                    limit = int(request.POST.get('limit') or 20)
                except Exception:
                    limit = 20
                limit = max(1, min(limit, 200))
                stats = _fetch_recent_emails(instance, limit=limit)
                msg = f"拉取完成：新增 {stats['fetched']} 封，跳过 {stats['skipped']} 封（已存在）"
                return redirect("/mailboxes?tested=1&ok=1&msg=" + urllib.parse.quote(msg))
            except Exception as e:
                return redirect("/mailboxes?tested=1&ok=0&msg=" + urllib.parse.quote(str(e)))
        # 保存（创建/更新）
        form = MailboxForm(request.POST, instance=instance)
        if form.is_valid():
            obj = form.save()
            return redirect(f"/mailboxes?mailbox_id={obj.id}&saved=1")
        else:
            error_msg = '表单校验失败，请检查必填项与字段格式。'
    else:
        form = MailboxForm(instance=instance)
        if request.GET.get('saved'):
            saved = True
        if request.GET.get('error'):
            error_msg = '操作失败（对象可能不存在或被引用，无法删除）。'
        if request.GET.get('tested'):
            test_ok = (request.GET.get('ok') == '1')
            test_msg = request.GET.get('msg') or ''

    # ---- 列表：简单筛选 + 分页 ----
    kw = (request.GET.get('kw') or '').strip()
    enabled_val = (request.GET.get('enabled') or '').strip()
    try:
        page = int(request.GET.get('page') or 1)
    except Exception:
        page = 1
    try:
        page_size = int(request.GET.get('page_size') or 10)
    except Exception:
        page_size = 10
    page_size = max(1, min(page_size, 100))

    mboxes_qs = Mailbox.objects.all()
    if kw:
        mboxes_qs = mboxes_qs.filter(Q(name__icontains=kw) | Q(username__icontains=kw) | Q(host__icontains=kw))
    if enabled_val in ('1', '0'):
        mboxes_qs = mboxes_qs.filter(enabled=(enabled_val == '1'))
    mboxes_qs = mboxes_qs.order_by('name')

    paginator = Paginator(mboxes_qs, page_size)
    page_obj = paginator.get_page(page)

    ctx = {
        'form': form,
        'mailboxes': page_obj.object_list,
        'page_obj': page_obj,
        'kw': kw,
        'enabled_selected': enabled_val,
        'page_size': page_size,
        'editing': instance is not None,
        'editing_mailbox': instance,
        'saved': saved,
        'error_msg': error_msg,
        'test_ok': test_ok,
        'test_msg': test_msg,
    }
    return render(request, 'mailboxes.html', ctx)


# ------------------ 运行/浏览 ------------------

def runs(request):
    recent = ExtractionRun.objects.select_related('scene').order_by('-id')[:20]

    # ----- 数据浏览器（查看/查询 + 分页）-----
    model_map = {
        'AIProvider': AIProvider,
        'Scene': Scene,
        'Mailbox': Mailbox,
        'EmailMessage': EmailMessage,
        'ExtractionRun': ExtractionRun,
        'ExtractionResult': ExtractionResult,
        'Shipment': Shipment,
    }
    browse_model_key = request.GET.get('browse_model') or 'Shipment'
    if browse_model_key not in model_map:
        browse_model_key = 'Shipment'
    model_cls = model_map[browse_model_key]

    q = (request.GET.get('q') or '').strip()
    try:
        page = int(request.GET.get('page') or 1)
    except Exception:
        page = 1
    try:
        page_size = int(request.GET.get('page_size') or 10)
    except Exception:
        page_size = 10
    page_size = max(1, min(page_size, 100))

    qs = model_cls.objects.all()
    # 简单搜索：对字符型字段做 icontains；若 q 为数字，则额外按 id 精确匹配
    if q:
        q_filter = Q()
        for f in model_cls._meta.fields:
            it = f.get_internal_type()
            if it in ('CharField', 'TextField', 'URLField', 'EmailField'):
                q_filter |= Q(**{f.name + '__icontains': q})
        if q.isdigit():
            q_filter |= Q(**{model_cls._meta.pk.name: int(q)})
        qs = qs.filter(q_filter)

    # 排序：默认按 id 倒序
    qs = qs.order_by('-' + model_cls._meta.pk.name)

    paginator = Paginator(qs, page_size)
    page_obj = paginator.get_page(page)

    # 为表格列出模型字段名（简单文本渲染，避免超长字段）
    field_names = [f.name for f in model_cls._meta.fields]
    def _fmt(v):
        if v is None:
            return ''
        s = str(v)
        return s if len(s) <= 120 else s[:117] + '...'

    rows = []
    for obj in page_obj.object_list:
        row = {}
        for fn in field_names:
            row[fn] = _fmt(getattr(obj, fn))
        rows.append(row)
    # 单封测试运行反馈
    single_ok = request.GET.get('single_ok')
    single_msg = request.GET.get('single_msg') or ''

    ctx = {
        'recent': recent,
        'recent_runs': recent,  # 模板兼容
        'browse_model_key': browse_model_key,
        'browse_models': list(model_map.keys()),
        'q': q,
        'browse_q': q,
        'page_obj': page_obj,
        'browse_page_obj': page_obj,
        'field_names': field_names,
        'browse_columns': field_names,
        'rows': rows,
        'browse_records': rows,
        'page_size': page_size,
        'browse_page_size': page_size,
        # 单封测试所需数据
        'mailboxes': Mailbox.objects.filter(enabled=True).order_by('name'),
        'scenes': Scene.objects.filter(enabled=True).order_by('priority', 'name'),
        'single_ok': single_ok,
        'single_msg': single_msg,
    }
    return render(request, 'runs.html', ctx)


def debug_trigger_merge(request):
    """
    调试用：手动触发一次 merge 流程。
    兼容：
      - 旧版：POST 传 scene_id、mailbox_id、email_id（均可选）
      - 新版：GET/POST 传 scene 名称（scene 或 scene_name），其他字段按需传
    """
    data = request.POST if request.method == 'POST' else request.GET

    # 解析场景：优先 scene_id，其次 scene 名称
    scene = None
    scene_id_raw = (data.get('scene_id') or '').strip()
    scene_name = (data.get('scene') or '').strip()
    if scene_id_raw.isdigit():
        scene = get_object_or_404(Scene, pk=int(scene_id_raw))
    elif scene_name:
        scene = Scene.objects.filter(name=scene_name).first()
        if not scene:
            return HttpResponseBadRequest('Scene not found')
    else:
        return HttpResponseBadRequest('Missing scene or scene_id')

    # 解析 mailbox/email：可选；未提供时取启用的第一个邮箱与最近一封邮件；都不存在时创建占位邮件
    mailbox = None
    email = None
    mailbox_id_raw = (data.get('mailbox_id') or '').strip()
    email_id_raw = (data.get('email_id') or '').strip()
    if mailbox_id_raw.isdigit():
        mailbox = get_object_or_404(Mailbox, pk=int(mailbox_id_raw))
    else:
        mailbox = Mailbox.objects.filter(enabled=True).order_by('id').first()
        if not mailbox:
            return HttpResponseBadRequest('No mailbox available')

    if email_id_raw.isdigit():
        email = get_object_or_404(EmailMessage, pk=int(email_id_raw))
    else:
        email = EmailMessage.objects.order_by('-id').first()
        if not email:
            # 创建占位邮件（用于调试合并逻辑）
            email = EmailMessage.objects.create(
                message_id=str(uuid.uuid4()),
                subject='DEBUG TRIGGER MERGE',
                from_addr='', to_addr='', raw_path='', body_text='debug', body_html='', attachments=[],
                mailbox=mailbox,
                status='pending'
            )

    # 从请求构造结果字典（仅收集提供的字段）
    candidate_keys = ['po_no', 'bl_no', 'booking_no', 'container_no', 'vessel', 'voyage', 'pol', 'pod', 'etd', 'eta', 'ata']
    results = {}
    for k in candidate_keys:
        v = (data.get(k) or '').strip()
        if v:
            results[k] = {'value': v}

    # 兜底：如完全未传字段，给出示例，避免空合并
    if not results:
        results = {'po_no': {'value': 'PO-DEBUG'}, 'booking_no': {'value': 'BK-DEBUG'}}

    outcome = process_extraction_and_merge(scene=scene, mailbox=mailbox, email=email, results=results, reasoning='debug_trigger_merge')

    return JsonResponse({'ok': True, 'outcome': outcome})


# ------------------ 单封测试运行辅助 ------------------

def _get_provider_api_key(provider: AIProvider) -> str:
    alias = provider.api_key_alias or ''
    if not alias:
        return ''
    return os.environ.get(f"AIPROVIDER_SECRET_{alias}", "")


def _render_prompt(scene: Scene, email_obj: EmailMessage) -> str:
    sent_date = email_obj.date.strftime('%Y-%m-%d') if email_obj.date else ''
    fields_csv = scene.fields_to_extract or ''
    base = scene.prompt_template or ''
    if base.strip():
        try:
            return base.format(
                email_body=email_obj.body_text or email_obj.body_html,  # 替换 {email_body}
                email_sent_date=sent_date  # 替换 {email_sent_date}
            )
            # return base.format(subject=email_obj.subject, body=email_obj.body_text or email_obj.body_html, sent_date=sent_date, fields=fields_csv)
        except Exception:
            pass
    # 默认提示模板（参考 test.py 思路，精简版）
    return (
        f"请从以下邮件的标题与正文中提取这些字段（以json返回，仅包含这些键）：{fields_csv}. "
        f"若日期缺少年份，以邮件发件日期补全年份；无法确定则留空。标题：{email_obj.subject}\n正文：{(email_obj.body_text or email_obj.body_html)[:4000]}\n"
    )


def _call_ai_provider(provider: AIProvider, prompt: str, timeout: int = None) -> dict:
    url = provider.base_url
    headers = dict(provider.extra_headers or {})
    key = _get_provider_api_key(provider)
    if provider.api_key_header_name:
        headers[provider.api_key_header_name] = (provider.api_key_prefix or '') + (key or '')
    headers.setdefault('Content-Type', 'application/json')
    body = {
        "messages": [{"role": "user", "content": prompt}],
        "stream": True,
        "temperature": 0.1,
        "top_p": 0.95,
        "top_k": 20,
        "min_p": 0.0,
        "max_tokens": 8192,
    }
    # if provider.model_name:
    #     body["model"] = provider.model_name,
    body_json = json.dumps(body) # 不能加ensure_ascii=False，否则会报错{"detail":"There was an error parsing the body"}
    # print(f"请求数据：{body_json}")
    # logger.info(f"📶 AI request to {url} (model={provider.model_name or '-'})")
    resp = requests.post(url, headers=headers, data=body_json, stream=True, timeout=(30, timeout or provider.timeout_seconds or 30))
    if resp.status_code != 200:
        raise RuntimeError(f"AI请求失败: {resp.status_code} {resp.text[:500]}")
    chunks = []
    chunks_reason = []
    for line in resp.iter_lines():
        if not line:
            continue
        try:
            s = line.decode('utf-8')
            if s.startswith('data:'):
                s = s[5:].strip()
            j = json.loads(s)
            delta = j.get('choices', [{}])[0].get('delta', {}).get('content', '')
            delta_reason= j.get("choices", [{}])[0].get("delta", {}).get("reasoning_content", "")
            if delta:
                chunks.append(delta)
            if delta_reason:
                chunks_reason.append(delta_reason)
        except Exception:
            continue
    text = ''.join(chunks).strip()
    reasoning_content = ''.join(chunks_reason).strip()
    # 去掉代码块围栏
    cleaned = text.replace('```', '').replace('json', '').strip()
    # 寻找 JSON 边界
    l = cleaned.find('{')
    r = cleaned.rfind('}')
    if l != -1 and r != -1 and r > l:
        cleaned = cleaned[l:r+1]
    try:
        # logger.info(f"AI返回结果(处理后)：{cleaned}\n思考过程:{reasoning_content}")
        return json.loads(cleaned), reasoning_content
    except Exception as e:
        logger.warning(f"AI返回解析失败，将返回原文供排查：{traceback.format_exc()}")
        raise
 
 
def _fetch_one_latest_email(mb: Mailbox) -> EmailMessage:
    imap = _imap_login(mb)
    try:
        imap.select(mb.folder or 'INBOX')
        # 取最新一封
        status, data = imap.search(None, 'ALL')
        if status != 'OK' or not data or not data[0]:
            raise RuntimeError('该邮箱文件夹无邮件')
        ids = data[0].split()
        latest_id = ids[-1]
        status, msg_data = imap.fetch(latest_id, '(RFC822)')
        if status != 'OK':
            raise RuntimeError('读取邮件失败')
        raw = msg_data[0][1]
        msg = email.message_from_bytes(raw)
        mid = msg.get('Message-Id') or msg.get('Message-ID') or ''
        if not mid:
            mid = str(uuid.uuid4())
        else:
            mid = mid.strip()
        subject = _decode_mime_header(msg.get('Subject'))
        from_addr = _decode_mime_header(msg.get('From'))
        to_addr = _decode_mime_header(msg.get('To'))
        try:
            dt = parsedate_to_datetime(msg.get('Date')) if msg.get('Date') else None
        except Exception:
            dt = None
        body_text, body_html, attach_list = _extract_bodies_and_attachments(msg, mb.id, False, forward_patterns=mb.forward_patterns)
        raw_path = ''
        obj = EmailMessage.objects.create(
            message_id=mid,
            subject=subject or '',
            from_addr=from_addr or '',
            to_addr=to_addr or '',
            date=dt,
            raw_path=raw_path or '',
            body_text=body_text or '',
            body_html=body_html or '',
            attachments=attach_list or [],
            mailbox=mb,
            status='pending'
        )
        return obj
    finally:
        try:
            imap.close()
        except Exception:
            pass
        try:
            imap.logout()
        except Exception:
            pass


def run_single(request):
    if request.method != 'POST':
        return HttpResponseBadRequest('Only POST allowed')
    try:
        mailbox_id = int(request.POST.get('mailbox_id') or '0')
        scene_id = int(request.POST.get('scene_id') or '0')
    except Exception:
        return HttpResponseBadRequest('Invalid ids')
    mailbox = get_object_or_404(Mailbox, pk=mailbox_id)
    scene = get_object_or_404(Scene, pk=scene_id)
    try:
        upload = request.FILES.get('email_file')
        if upload:
            # 解析上传的 .eml 文件
            raw_bytes = upload.read() or b''
            if not raw_bytes:
                return HttpResponseBadRequest('上传文件为空')
            msg = email.message_from_bytes(raw_bytes)
            mid = (msg.get('Message-Id') or msg.get('Message-ID') or '').strip()
            if not mid:
                mid = str(uuid.uuid4())
            subject = _decode_mime_header(msg.get('Subject'))
            from_addr = _decode_mime_header(msg.get('From'))
            to_addr = _decode_mime_header(msg.get('To'))
            try:
                dt = parsedate_to_datetime(msg.get('Date')) if msg.get('Date') else None
            except Exception:
                dt = None
            body_text, body_html, attach_list = _extract_bodies_and_attachments(
                msg, mailbox.id, False, forward_patterns=mailbox.forward_patterns
            )
            raw_path = ''
            try:
                email_obj = EmailMessage.objects.get(message_id=mid)
            except EmailMessage.DoesNotExist:
                email_obj = EmailMessage.objects.create(
                    message_id=mid,
                    subject=subject or '',
                    from_addr=from_addr or '',
                    to_addr=to_addr or '',
                    date=dt,
                    raw_path=raw_path or '',
                    body_text=body_text or '',
                    body_html=body_html or '',
                    attachments=attach_list or [],
                    mailbox=mailbox,
                    status='pending'
                )
        else:
            # 走原逻辑：拉取最新一封
            email_obj = _fetch_one_latest_email(mailbox)

        # 先按主题检查委托单号（WT14位）
        subj = (email_obj.subject or '').strip()
        po_matches = re.findall(r'WT\d{14}', subj)
        if not po_matches:
            run = ExtractionRun.objects.create(scene=scene, mailbox=mailbox, email=email_obj, status='success')
            run.reasoning = '主题无委托单号'
            run.finished_at = timezone.now()
            run.save(update_fields=['status', 'finished_at', 'reasoning'])
            msg = f"单封测试完成：run#{run.id} status={run.status} updated=False updated_fields= reason=主题无委托单号"
            return redirect(f"/runs?single_ok=1&single_msg={urllib.parse.quote(msg)}")

        po_no = po_matches[0]
        # 渲染提示词并调用 AI
        prompt = _render_prompt(scene, email_obj)
        logger.debug(f"prompt: {prompt[:5000]}")
        ai_data, reasoning = _call_ai_provider(scene.ai_provider, prompt)
        # 字段映射（中文 -> 英文模型字段）
        key_map = {
            '船名': 'vessel',
            '航次': 'voyage',
            '箱号': 'container_no',
            'MBL': 'bl_no',
            'HBL': 'hbl_no',
            'ETD': 'etd',
            'ETA': 'eta',
            'ATA': 'ata',
            '委托单号': 'po_no',
            '业务委托号': 'po_no',
            '提货日期': 'pickup_date',
            'PO': 'po_no',
        }
        results = {
            'po_no': {'value': po_no, 'confidence': 1.0, 'source': 'regex'}
        }
        for k, v in (ai_data or {}).items():
            mk = key_map.get(k, k)
            # TODO: 改成使用scene中配置的应提取的字段
            if mk not in scene.fields_to_extract:
                continue
            if mk == 'po_no':
                # 以主题提取的委托单号为准，不被覆盖
                continue
            results[mk] = {'value': v or ''}
        print("results: ", results)
        outcome = process_extraction_and_merge(scene=scene, mailbox=mailbox, email=email_obj, results=results, reasoning=reasoning)
        msg = f"单封测试完成：run#{outcome.get('run_id')} status={outcome.get('status')} updated={outcome.get('updated')} updated_fields={','.join(outcome.get('updated_fields', []))}"
        return redirect(f"/runs?single_ok=1&single_msg={urllib.parse.quote(msg)}")
    except Exception as e:
        return redirect(f"/runs?single_ok=0&single_msg={urllib.parse.quote(str(e))}")

def _select_scene_by_subject(email_obj: EmailMessage) -> Optional[Scene]:
    """按 Scene.email_type 中配置的正则表达式匹配主题，命中即返回该场景（按 priority/name 顺序）。"""
    subject = (email_obj.subject or '')
    candidates = Scene.objects.filter(enabled=True).order_by('priority', 'name')
    for sc in candidates:
        pattern_text = (sc.email_type or '').strip()
        if not pattern_text:
            continue
        parts = [p.strip() for p in re.split(r'[\n,]+', pattern_text) if p.strip()]
        for pat in parts:
            try:
                if re.search(pat, subject, flags=re.IGNORECASE):
                    return sc
            except re.error:
                continue
    return None